package cn.lzd.demo1

import java.net.URL

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
  * Created by lzd on 2018/12/26.
  * http://hadoop.com/laozhao 统计hadoop这门课人名出现了几次
  */
object Demo1 {
  def main(args: Array[String]): Unit = {
    val sc = new SparkContext(new SparkConf().setAppName("demo1").setMaster("local"))
    val map: RDD[String] = sc.textFile("F:\\idea_work_me\\spider\\spark-demo\\src\\main\\resources\\demo2\\text").map(x => {
      val url = new URL(x)
      val host = url.getHost
      val project = host.substring(0, host.indexOf("."))
      val path = url.getPath.substring(1)
      project + ":" + path
    })

    val reduceByKey: RDD[(String, Int)] = map.map((_,1)).reduceByKey(_+_)

    val groupBy: RDD[(String, Iterable[(String, Int)])] = reduceByKey.groupBy(_._1.split(":")(0))

    val values: RDD[(String, List[(String, Int)])] = groupBy.mapValues(_.toList.sortBy(_._2).reverse)

    println(values.collect().toBuffer)
  }
}
